{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use natural language toolkit\n",
    "import nltk\n",
    "from nltk.stem.lancaster import LancasterStemmer\n",
    "import os\n",
    "import json\n",
    "import datetime\n",
    "stemmer = LancasterStemmer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 sentences in training data\n"
     ]
    }
   ],
   "source": [
    "# 3 classes of training data\n",
    "training_data = []\n",
    "training_data.append({\"class\":\"greeting\", \"sentence\":\"how are you?\"})\n",
    "training_data.append({\"class\":\"greeting\", \"sentence\":\"how is your day?\"})\n",
    "training_data.append({\"class\":\"greeting\", \"sentence\":\"good day\"})\n",
    "training_data.append({\"class\":\"greeting\", \"sentence\":\"how is it going today?\"})\n",
    "\n",
    "training_data.append({\"class\":\"goodbye\", \"sentence\":\"have a nice day\"})\n",
    "training_data.append({\"class\":\"goodbye\", \"sentence\":\"see you later\"})\n",
    "training_data.append({\"class\":\"goodbye\", \"sentence\":\"have a nice day\"})\n",
    "training_data.append({\"class\":\"goodbye\", \"sentence\":\"talk to you soon\"})\n",
    "\n",
    "training_data.append({\"class\":\"sandwich\", \"sentence\":\"make me a sandwich\"})\n",
    "training_data.append({\"class\":\"sandwich\", \"sentence\":\"can you make a sandwich?\"})\n",
    "training_data.append({\"class\":\"sandwich\", \"sentence\":\"having a sandwich today?\"})\n",
    "training_data.append({\"class\":\"sandwich\", \"sentence\":\"what's for lunch?\"})\n",
    "print (\"%s sentences in training data\" % len(training_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 documents\n",
      "3 classes ['greeting', 'goodbye', 'sandwich']\n",
      "26 unique stemmed words ['sandwich', 'hav', 'a', 'how', 'for', 'ar', 'good', 'mak', 'me', 'it', 'day', 'soon', 'nic', 'lat', 'going', 'you', 'today', 'can', 'lunch', 'is', \"'s\", 'see', 'to', 'talk', 'yo', 'what']\n"
     ]
    }
   ],
   "source": [
    "words = []\n",
    "classes = []\n",
    "documents = []\n",
    "ignore_words = ['?']\n",
    "# loop through each sentence in our training data\n",
    "for pattern in training_data:\n",
    "    # tokenize each word in the sentence\n",
    "    w = nltk.word_tokenize(pattern['sentence'])\n",
    "    # add to our words list\n",
    "    words.extend(w)\n",
    "    # add to documents in our corpus\n",
    "    documents.append((w, pattern['class']))\n",
    "    # add to our classes list\n",
    "    if pattern['class'] not in classes:\n",
    "        classes.append(pattern['class'])\n",
    "\n",
    "# stem and lower each word and remove duplicates\n",
    "words = [stemmer.stem(w.lower()) for w in words if w not in ignore_words]\n",
    "words = list(set(words))\n",
    "\n",
    "# remove duplicates\n",
    "classes = list(set(classes))\n",
    "\n",
    "print (len(documents), \"documents\")\n",
    "print (len(classes), \"classes\", classes)\n",
    "print (len(words), \"unique stemmed words\", words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# words 26\n",
      "# classes 3\n"
     ]
    }
   ],
   "source": [
    "# create our training data\n",
    "training = []\n",
    "output = []\n",
    "# create an empty array for our output\n",
    "output_empty = [0] * len(classes)\n",
    "\n",
    "# training set, bag of words for each sentence\n",
    "for doc in documents:\n",
    "    # initialize our bag of words\n",
    "    bag = []\n",
    "    # list of tokenized words for the pattern\n",
    "    pattern_words = doc[0]\n",
    "    # stem each word\n",
    "    pattern_words = [stemmer.stem(word.lower()) for word in pattern_words]\n",
    "    # create our bag of words array\n",
    "    for w in words:\n",
    "        bag.append(1) if w in pattern_words else bag.append(0)\n",
    "\n",
    "    training.append(bag)\n",
    "    # output is a '0' for each tag and '1' for current tag\n",
    "    output_row = list(output_empty)\n",
    "    output_row[classes.index(doc[1])] = 1\n",
    "    output.append(output_row)\n",
    "\n",
    "print (\"# words\", len(words))\n",
    "print (\"# classes\", len(classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['how', 'ar', 'you', '?']\n",
      "[0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "[1, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "# sample training/output\n",
    "i = 0\n",
    "w = documents[i][0]\n",
    "print ([stemmer.stem(word.lower()) for word in w])\n",
    "print (training[i])\n",
    "print (output[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "\n",
    "# compute sigmoid nonlinearity\n",
    "def sigmoid(x):\n",
    "    output = 1/(1+np.exp(-x))\n",
    "    return output\n",
    "\n",
    "# convert output of sigmoid function to its derivative\n",
    "def sigmoid_output_to_derivative(output):\n",
    "    return output*(1-output)\n",
    " \n",
    "def clean_up_sentence(sentence):\n",
    "    # tokenize the pattern\n",
    "    sentence_words = nltk.word_tokenize(sentence)\n",
    "    # stem each word\n",
    "    sentence_words = [stemmer.stem(word.lower()) for word in sentence_words]\n",
    "    return sentence_words\n",
    "\n",
    "# return bag of words array: 0 or 1 for each word in the bag that exists in the sentence\n",
    "def bow(sentence, words, show_details=False):\n",
    "    # tokenize the pattern\n",
    "    sentence_words = clean_up_sentence(sentence)\n",
    "    # bag of words\n",
    "    bag = [0]*len(words)  \n",
    "    for s in sentence_words:\n",
    "        for i,w in enumerate(words):\n",
    "            if w == s: \n",
    "                bag[i] = 1\n",
    "                if show_details:\n",
    "                    print (\"found in bag: %s\" % w)\n",
    "\n",
    "    return(np.array(bag))\n",
    "\n",
    "def think(sentence, show_details=False):\n",
    "    x = bow(sentence.lower(), words, show_details)\n",
    "    if show_details:\n",
    "        print (\"sentence:\", sentence, \"\\n bow:\", x)\n",
    "    # input layer is our bag of words\n",
    "    l0 = x\n",
    "    # matrix multiplication of input and hidden layer\n",
    "    l1 = sigmoid(np.dot(l0, synapse_0))\n",
    "    # output layer\n",
    "    l2 = sigmoid(np.dot(l1, synapse_1))\n",
    "    return l2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ANN and Gradient Descent code from https://iamtrask.github.io//2015/07/27/python-network-part2/\n",
    "def train(X, y, hidden_neurons=10, alpha=1, epochs=50000, dropout=False, dropout_percent=0.5):\n",
    "\n",
    "    print (\"Training with %s neurons, alpha:%s, dropout:%s %s\" % (hidden_neurons, str(alpha), dropout, dropout_percent if dropout else '') )\n",
    "    print (\"Input matrix: %sx%s    Output matrix: %sx%s\" % (len(X),len(X[0]),1, len(classes)) )\n",
    "    np.random.seed(1)\n",
    "\n",
    "    last_mean_error = 1\n",
    "    # randomly initialize our weights with mean 0\n",
    "    synapse_0 = 2*np.random.random((len(X[0]), hidden_neurons)) - 1\n",
    "    synapse_1 = 2*np.random.random((hidden_neurons, len(classes))) - 1\n",
    "\n",
    "    prev_synapse_0_weight_update = np.zeros_like(synapse_0)\n",
    "    prev_synapse_1_weight_update = np.zeros_like(synapse_1)\n",
    "\n",
    "    synapse_0_direction_count = np.zeros_like(synapse_0)\n",
    "    synapse_1_direction_count = np.zeros_like(synapse_1)\n",
    "        \n",
    "    for j in iter(range(epochs+1)):\n",
    "\n",
    "        # Feed forward through layers 0, 1, and 2\n",
    "        layer_0 = X\n",
    "        layer_1 = sigmoid(np.dot(layer_0, synapse_0))\n",
    "                \n",
    "        if(dropout):\n",
    "            layer_1 *= np.random.binomial([np.ones((len(X),hidden_neurons))],1-dropout_percent)[0] * (1.0/(1-dropout_percent))\n",
    "\n",
    "        layer_2 = sigmoid(np.dot(layer_1, synapse_1))\n",
    "\n",
    "        # how much did we miss the target value?\n",
    "        layer_2_error = y - layer_2\n",
    "\n",
    "        if (j% 10000) == 0 and j > 5000:\n",
    "            # if this 10k iteration's error is greater than the last iteration, break out\n",
    "            if np.mean(np.abs(layer_2_error)) < last_mean_error:\n",
    "                print (\"delta after \"+str(j)+\" iterations:\" + str(np.mean(np.abs(layer_2_error))) )\n",
    "                last_mean_error = np.mean(np.abs(layer_2_error))\n",
    "            else:\n",
    "                print (\"break:\", np.mean(np.abs(layer_2_error)), \">\", last_mean_error )\n",
    "                break\n",
    "                \n",
    "        # in what direction is the target value?\n",
    "        # were we really sure? if so, don't change too much.\n",
    "        layer_2_delta = layer_2_error * sigmoid_output_to_derivative(layer_2)\n",
    "\n",
    "        # how much did each l1 value contribute to the l2 error (according to the weights)?\n",
    "        layer_1_error = layer_2_delta.dot(synapse_1.T)\n",
    "\n",
    "        # in what direction is the target l1?\n",
    "        # were we really sure? if so, don't change too much.\n",
    "        layer_1_delta = layer_1_error * sigmoid_output_to_derivative(layer_1)\n",
    "        \n",
    "        synapse_1_weight_update = (layer_1.T.dot(layer_2_delta))\n",
    "        synapse_0_weight_update = (layer_0.T.dot(layer_1_delta))\n",
    "        \n",
    "        if(j > 0):\n",
    "            synapse_0_direction_count += np.abs(((synapse_0_weight_update > 0)+0) - ((prev_synapse_0_weight_update > 0) + 0))\n",
    "            synapse_1_direction_count += np.abs(((synapse_1_weight_update > 0)+0) - ((prev_synapse_1_weight_update > 0) + 0))        \n",
    "        \n",
    "        synapse_1 += alpha * synapse_1_weight_update\n",
    "        synapse_0 += alpha * synapse_0_weight_update\n",
    "        \n",
    "        prev_synapse_0_weight_update = synapse_0_weight_update\n",
    "        prev_synapse_1_weight_update = synapse_1_weight_update\n",
    "\n",
    "    now = datetime.datetime.now()\n",
    "\n",
    "    # persist synapses\n",
    "    synapse = {'synapse0': synapse_0.tolist(), 'synapse1': synapse_1.tolist(),\n",
    "               'datetime': now.strftime(\"%Y-%m-%d %H:%M\"),\n",
    "               'words': words,\n",
    "               'classes': classes\n",
    "              }\n",
    "    synapse_file = \"synapses.json\"\n",
    "\n",
    "    with open(synapse_file, 'w') as outfile:\n",
    "        json.dump(synapse, outfile, indent=4, sort_keys=True)\n",
    "    print (\"saved synapses to:\", synapse_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with 20 neurons, alpha:0.1, dropout:False \n",
      "Input matrix: 12x26    Output matrix: 1x3\n",
      "delta after 10000 iterations:0.0062613597435\n",
      "delta after 20000 iterations:0.00428296074919\n",
      "delta after 30000 iterations:0.00343930779307\n",
      "delta after 40000 iterations:0.00294648034566\n",
      "delta after 50000 iterations:0.00261467859609\n",
      "delta after 60000 iterations:0.00237219554105\n",
      "delta after 70000 iterations:0.00218521899378\n",
      "delta after 80000 iterations:0.00203547284581\n",
      "delta after 90000 iterations:0.00191211022401\n",
      "delta after 100000 iterations:0.00180823798397\n",
      "saved synapses to: synapses.json\n",
      "processing time: 6.501226902008057 seconds\n"
     ]
    }
   ],
   "source": [
    "X = np.array(training)\n",
    "y = np.array(output)\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "train(X, y, hidden_neurons=20, alpha=0.1, epochs=100000, dropout=False, dropout_percent=0.2)\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "print (\"processing time:\", elapsed_time, \"seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sudo make me a sandwich \n",
      " classification: [['sandwich', 0.99917711814437993]]\n",
      "how are you today? \n",
      " classification: [['greeting', 0.99864563257858363]]\n",
      "talk to you tomorrow \n",
      " classification: [['goodbye', 0.95647479275905511]]\n",
      "who are you? \n",
      " classification: [['greeting', 0.8964283843977312]]\n",
      "make me some lunch \n",
      " classification: [['sandwich', 0.95371924052636048]]\n",
      "\n",
      "found in bag: how\n",
      "found in bag: yo\n",
      "found in bag: lunch\n",
      "sentence: how was your lunch? \n",
      " bow: [0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0]\n",
      "how was your lunch? \n",
      " classification: [['greeting', 0.97947197384371776]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[['greeting', 0.97947197384371776]]"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# probability threshold\n",
    "ERROR_THRESHOLD = 0.2\n",
    "# load our calculated synapse values\n",
    "synapse_file = 'synapses.json' \n",
    "with open(synapse_file) as data_file: \n",
    "    synapse = json.load(data_file) \n",
    "    synapse_0 = np.asarray(synapse['synapse0']) \n",
    "    synapse_1 = np.asarray(synapse['synapse1'])\n",
    "\n",
    "def classify(sentence, show_details=False):\n",
    "    results = think(sentence, show_details)\n",
    "\n",
    "    results = [[i,r] for i,r in enumerate(results) if r>ERROR_THRESHOLD ] \n",
    "    results.sort(key=lambda x: x[1], reverse=True) \n",
    "    return_results =[[classes[r[0]],r[1]] for r in results]\n",
    "    print (\"%s \\n classification: %s\" % (sentence, return_results))\n",
    "    return return_results\n",
    "\n",
    "classify(\"sudo make me a sandwich\")\n",
    "classify(\"how are you today?\")\n",
    "classify(\"talk to you tomorrow\")\n",
    "classify(\"who are you?\")\n",
    "classify(\"make me some lunch\")\n",
    "print ()\n",
    "classify(\"how was your lunch?\", show_details=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
